{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 简介\n",
    "爬取京东某本书的评论，然后进行分析，选择的书为：《ES6标准入门（第3版）》 、阮一峰 著 、博文视点出品、（电子工业出版社 书店）。\n",
    "该书有2.1万评论，其它书店该书评论寥寥无几，进行数据分析一探究竟。\n",
    "\n",
    "地址：https://item.jd.com/12172449.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "```\n",
    "    唯一变量为page\n",
    "```\n",
    "# page_num = 0\n",
    "# url = \"https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=12172449&score=0&sortType=5&page=\"+page_num+\n",
    "# \"&pageSize=10&isShadowSku=0&rid=0&fold=0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 爬取评论&数据清洗"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-22T05:13:28.109254Z",
     "start_time": "2020-08-22T05:13:27.864668Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import re\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-22T05:13:30.462741Z",
     "start_time": "2020-08-22T05:13:30.168960Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "headers1 = {\n",
    "    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "    'accept-encoding': 'gzip, deflate, br',\n",
    "    'accept-language': 'zh-CN,zh;q=0.9',\n",
    "    'cache-control': 'no-cache',\n",
    "    # 'cookie':  'UM_distinctid=172b674e5f71cb-034c5dfe424061-3a365305-1fa400-172b674e5f8848; PHPSESSID=4gsduu0v6tmi1lkbsf8cp7kej0; local_zone=320100%7Cnanjing%7C%C4%CF%BE%A9%7C58238; Hm_lvt_d64174522c86449826babe56fb2a88ff=1592199015; CNZZDATA1089612=cnzz_eid%3D725595362-1592200519-https%253A%252F%252Fguilin.cncn.com%252F%26ntime%3D1592200519; Hm_lpvt_d64174522c86449826babe56fb2a88ff=1592203476',\n",
    "    'pragma': 'no-cache',\n",
    "    # 'referer':  'https://beihai.cncn.com/jingdian/beihai/',\n",
    "    'sec-fetch-dest': 'document',\n",
    "    'sec-fetch-mode': 'navigate',\n",
    "    'sec-fetch-site': 'same-site',\n",
    "    'sec-fetch-user': '?1',\n",
    "    'upgrade-insecure-requests': '1',\n",
    "    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',\n",
    "\n",
    "}\n",
    "\n",
    "headers = {\n",
    "    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "    'accept-encoding': 'gzip, deflate, br',\n",
    "    'accept-language': 'zh-CN,zh;q=0.9',\n",
    "    'cache-control': 'no-cache',\n",
    "    # 'cookie':  'UM_distinctid=172b674e5f71cb-034c5dfe424061-3a365305-1fa400-172b674e5f8848; PHPSESSID=4gsduu0v6tmi1lkbsf8cp7kej0; local_zone=320100%7Cnanjing%7C%C4%CF%BE%A9%7C58238; Hm_lvt_d64174522c86449826babe56fb2a88ff=1592199015; CNZZDATA1089612=cnzz_eid%3D725595362-1592200519-https%253A%252F%252Fguilin.cncn.com%252F%26ntime%3D1592200519; Hm_lpvt_d64174522c86449826babe56fb2a88ff=1592203476',\n",
    "    'pragma': 'no-cache',\n",
    "    # 'referer':  'https://beihai.cncn.com/jingdian/beihai/',\n",
    "    'sec-fetch-dest': 'document',\n",
    "    'sec-fetch-mode': 'navigate',\n",
    "    'sec-fetch-site': 'same-site',\n",
    "    'sec-fetch-user': '?1',\n",
    "    'upgrade-insecure-requests': '1',\n",
    "    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',\n",
    "\n",
    "}\n",
    "\n",
    "\n",
    "def run():\n",
    "    ```\n",
    "    正则表达式匹配串\n",
    "    content: 评论\n",
    "    score: 星级\n",
    "    creationTime: 评论时间\n",
    "    ```\n",
    "    content = re.compile(r\"content\\\":\\\"\\S*\\\",\\\"creationTime\")\n",
    "    score = re.compile(\"score\\\":[0-5]\")\n",
    "    creationTime = re.compile(\"[0-9]*-[0-9]*-[0-9]*\\s[0-9]*:[0-9]*:[0-9]*\")\n",
    "    \n",
    "    \n",
    "    ```\n",
    "    总共有 100页，只提取主评论（追加评论不提，因为没有star）\n",
    "    每页10个评论 \n",
    "    官网显示总共2.1w评论，1.6w默认\n",
    "    提取数据为652条\n",
    "    ```\n",
    "    for i in range(0,99):\n",
    "        page_num = str(i)\n",
    "        q_url = \"https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=12172449&score=0&sortType=5&page=\"+page_num+\"&pageSize=10&isShadowSku=0&rid=0&fold=0\"\n",
    "\n",
    "        # 获取评论json数据\n",
    "        url_first = q_url\n",
    "        html = requests.get(url_first, headers1)\n",
    "\n",
    "        # 获取基本数据集\n",
    "        text = []\n",
    "        star = []\n",
    "        time = []\n",
    "        text = content.findall(html.text)\n",
    "        star = score.findall(html.text)\n",
    "        time = creationTime.findall(html.text)\n",
    "\n",
    "        # 数据清理\n",
    "        star_ = []\n",
    "        time_ = []\n",
    "        text_ = []\n",
    "        for k in range(0,len(text)):\n",
    "            star_.append(star[k][-1:])\n",
    "            time_.append(time[k])\n",
    "            text_.append(text[k][10:-15])\n",
    "\n",
    "        # 保存为csv\n",
    "        data = {'star':star_,'time':time_,'text':text_}\n",
    "        file_name = 'data.csv'\n",
    "        df = pd.DataFrame(data)\n",
    "        if os.path.exists(file_name):\n",
    "            df.to_csv(file_name, mode='a', index=False, na_rep='NA',header=False, encoding='utf-8')\n",
    "        else:\n",
    "            df.to_csv(file_name, mode='a', index=False, na_rep='NA',header=['score', 'creationTime', 'content'], encoding='utf-8')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    data = run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据分析\n",
    "使用 jieba 分词对一个文本进行分词，统计次数出现最多的词语"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "\n",
    "txt = open(\"data.csv\", \"r\", encoding='utf-8').read()\n",
    "words = jieba.lcut(txt)     # 使用精确模式对文本进行分词\n",
    "counts = {}     # 通过键值对的形式存储词语及其出现的次数\n",
    "\n",
    "for word in words:\n",
    "    if len(word) == 1:    # 单个词语不计算在内\n",
    "        continue\n",
    "    else:\n",
    "        counts[word] = counts.get(word, 0) + 1    # 遍历所有词语，每出现一次其对应的值加 1\n",
    "\n",
    "items = list(counts.items())\n",
    "items.sort(key=lambda x: x[1], reverse=True)    # 根据词语出现的次数进行从大到小排序\n",
    "\n",
    "for i in range(3):\n",
    "    word, count = items[i]\n",
    "    print(\"{0:<5}{1:>5}\".format(word, count))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试 [LAC](https://github.com/baidu/lac)百度中文词法分析（分词+词性+专名）系统"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T03:09:19.359475Z",
     "start_time": "2020-08-28T03:09:19.293033Z"
    }
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'LAC'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-20c823d8055f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mLAC\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLAC\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# 装载分词模型\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mlac\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLAC\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'seg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'LAC'"
     ]
    }
   ],
   "source": [
    "from LAC import LAC\n",
    "\n",
    "# 装载分词模型\n",
    "lac = LAC(mode='seg')\n",
    "\n",
    "# 单个样本输入，输入为Unicode编码的字符串\n",
    "text = u\"LAC是个优秀的分词工具\"\n",
    "seg_result = lac.run(text)\n",
    "\n",
    "# 批量样本输入, 输入为多个句子组成的list，平均速率会更快\n",
    "texts = [u\"LAC是个优秀的分词工具\", u\"百度是一家高科技公司\"]\n",
    "seg_result = lac.run(texts)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
